#!/bin/bash

ROOT_PATH=`cd $(dirname $0) && pwd`
SCRIPT_NAME=`basename $0`
SCRIPT=$ROOT_PATH/`basename $0`
cd ${ROOT_PATH}/softwares/OpenCAEPoro/optparamsdpcoptintel

. /usr/share/modules/init/bash
module use /opt/intel/modulefiles
module use /opt/nvidia/hpc_sdk/modulefiles
module use /opt/modulefiles
module use ${ROOT_PATH}/env

module purge
module load compiler/2023.1.0
module load mpi/2021.11
module load mkl/2023.1.0

# Load hypre / parmetis / petsc / jemalloc
module load hypre/2.28.0/optparamsdpcoptintel parmetis/4.0.3/optparamsdpcoptintel petsc/3.19.3/optparamsdpcoptintel jemalloc/5.3.0/optparamsdpcoptintel

export OMP_NUM_THREADS=2
export I_MPI_PIN_DOMAIN=omp
export I_MPI_FABRICS=shm:ofi
export I_MPI_PIN_ORDER=scatter
export I_MPI_PIN_CELL=core

log_t=`date "+%Y%m%d-%H%M%S"`

mkdir -p logs/${log_t}

touch logs/${log_t}/start.sh
cat $SCRIPT > logs/${log_t}/start.sh

mpirun -hosts 198.19.19.254,198.19.19.205 -ppn 24 -np 48 -print-rank-map -iface ib0 ./testOpenCAEPoro ./data/case1/case1.data verbose=1 2>&1 | tee logs/${log_t}/run.log

mv ./data/case1/FastReview.out logs/${log_t}/
mv ./data/case1/statistics.out logs/${log_t}/
mv ./data/case1/SUMMARY.out logs/${log_t}/